{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据质量分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "缺失值<br>\n",
    "异常值<br>\n",
    "重复值<br>\n",
    "一致性<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 缺失值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jack</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7300</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lily</td>\n",
       "      <td>21.0</td>\n",
       "      <td>11000</td>\n",
       "      <td>python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tom</td>\n",
       "      <td>25.0</td>\n",
       "      <td>12000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Joe</td>\n",
       "      <td>81.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Jack</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   name   age  salary    lang\n",
       "0  Jack  18.0    8000  Python\n",
       "1  Mary   NaN    7300    Java\n",
       "2  Lily  21.0   11000  python\n",
       "3   Tom  25.0   12000     NaN\n",
       "4   Joe  81.0      -1     NaN\n",
       "5  Jack  18.0    8000  Python"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1、isnull()   &  notnull()    --  判断是否为NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    name    age  salary   lang\n",
       "0  False  False   False  False\n",
       "1  False   True   False  False\n",
       "2  False  False   False  False\n",
       "3  False  False   False   True\n",
       "4  False  False   False   True\n",
       "5  False  False   False  False"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name      0\n",
       "age       1\n",
       "salary    0\n",
       "lang      2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name      6\n",
       "age       5\n",
       "salary    6\n",
       "lang      4\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.notnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "isnull()只会将NaN判断为null，空值不会判断为null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jack</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7300</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lily</td>\n",
       "      <td>21.0</td>\n",
       "      <td>11000</td>\n",
       "      <td>python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tom</td>\n",
       "      <td>25.0</td>\n",
       "      <td>12000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Joe</td>\n",
       "      <td>81.0</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Jack</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   name   age  salary    lang\n",
       "0  Jack  18.0    8000  Python\n",
       "1  Mary   NaN    7300    Java\n",
       "2  Lily  21.0   11000  python\n",
       "3   Tom  25.0   12000     NaN\n",
       "4   Joe  81.0      -1        \n",
       "5  Jack  18.0    8000  Python"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[4, 'lang'] = ''\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    name    age  salary   lang\n",
       "0  False  False   False  False\n",
       "1  False   True   False  False\n",
       "2  False  False   False  False\n",
       "3  False  False   False   True\n",
       "4  False  False   False  False\n",
       "5  False  False   False  False"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull()          "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2、判断是否为空值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3    False\n",
       "4     True\n",
       "5    False\n",
       "Name: lang, dtype: bool"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['lang'] == ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df['lang'] == '')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 异常值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>32.600000</td>\n",
       "      <td>7716.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>27.208455</td>\n",
       "      <td>4219.567928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>18.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>18.000000</td>\n",
       "      <td>7475.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>21.000000</td>\n",
       "      <td>8000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>25.000000</td>\n",
       "      <td>10250.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>81.000000</td>\n",
       "      <td>12000.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             age        salary\n",
       "count   5.000000      6.000000\n",
       "mean   32.600000   7716.500000\n",
       "std    27.208455   4219.567928\n",
       "min    18.000000     -1.000000\n",
       "25%    18.000000   7475.000000\n",
       "50%    21.000000   8000.000000\n",
       "75%    25.000000  10250.000000\n",
       "max    81.000000  12000.000000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3    False\n",
       "4     True\n",
       "5    False\n",
       "Name: age, dtype: bool"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.age<16) | (df.age>60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum((df.age<16) | (df.age>60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x117b9b9b0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEkFJREFUeJzt3X+sX/V93/Hna3YIlCZAQnVFbTQjxepM3EVjd4Qq2nQdb2AgqvkjzUBT4zJrljbKsh9Sa+aqrAmWQJlG07XJ5MY0pssMjKYCDQa1yP2q2jQIkKQJ4Ka5glDsQUhioHVo09h974/7cfKNP/di+37v9dcuz4d05XPe5/M553OkL3758znne0lVIUnSsL817gFIkk49hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6y8c9gIU6//zza9WqVeMehtT57ne/y9lnnz3uYUhzevLJJ79dVT9xrHanbTisWrWKJ554YtzDkDqDwYCpqalxD0OaU5Lnj6edy0qSpI7hIEnqGA6SpI7hIEnqGA6SpM4xwyHJHUleTvLUUO3jSf44yVeS/H6Sc4eO3ZRkJsnXklwxVN/QajNJtg7VL0ryWKvfneSMxbxBSdKJO56Zw2eADUfV9gBrq+rvAn8C3ASQ5GLgWuDdrc8nkyxLsgz4LeBK4GLgutYW4Dbg9qp6F/AKsHmkO5IkjeyY4VBVfwgcOKr2B1V1qO0+Cqxs2xuBu6rqe1X1HDADXNp+Zqrq2ar6K+AuYGOSAO8H7m39dwHXjHhPkqQRLcaX4P45cHfbXsFsWByxr9UAXjiq/l7gncCrQ0Ez3L6TZAuwBWBiYoLBYDDq2KVjWrdu3Um5zvT09Em5jnQ8RgqHJNuAQ8BnF2c4b6yqdgA7ACYnJ8tvoepkqKoTar9q6wN849arl2g00smx4HBI8gvAB4D19cP/evYDFw41W9lqzFP/DnBukuVt9jDcXpI0Jgt6lTXJBuCXgJ+tqteHDt0PXJvkrUkuAlYDXwAeB1a3N5POYPah9f0tVKaBD7b+m4D7FnYrkqTFcjyvsu4G/i/wU0n2JdkM/CbwNmBPki8n+a8AVfU0cA/wDPAQcENVHW6zgl8EHgb2Ave0tgC/DPy7JDPMPoPYuah3KEk6YcdcVqqq6+Yoz/sXeFVtB7bPUX8QeHCO+rPMvs0kSTpF+A1pSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVJn+bgHIJ1M7/m1P+C1v/j+kl9n1dYHlvT855z1Fv7o5suX9Bp6cztmOCS5A/gA8HJVrW21dwB3A6uAbwAfqqpXkgT4BHAV8DrwC1X1xdZnE/Ar7bS3VNWuVv/7wGeAs4AHgY9UVS3S/Uk/4rW/+D7fuPXqJb3GYDBgampqSa+x1OEjHc+y0meADUfVtgKPVNVq4JG2D3AlsLr9bAE+BT8Ik5uB9wKXAjcnOa/1+RTwL4b6HX0tSdJJdsxwqKo/BA4cVd4I7Grbu4Brhup31qxHgXOTXABcAeypqgNV9QqwB9jQjr29qh5ts4U7h84lSRqThT5zmKiqF9v2S8BE214BvDDUbl+rvVF93xz1OSXZwuyMhImJCQaDwQKHrzezpf7cHDx48KR8Nv38aymN/EC6qirJSXlGUFU7gB0Ak5OTtdTruvob6KEHlvx5wMl45nAy7kNvbgt9lfWbbUmI9ufLrb4fuHCo3cpWe6P6yjnqkqQxWmg43A9satubgPuG6h/OrMuA19ry08PA5UnOaw+iLwcebsf+LMll7U2nDw+dS5I0JsfzKutuYAo4P8k+Zt86uhW4J8lm4HngQ635g8y+xjrD7Kus1wNU1YEkHwMeb+0+WlVHHnL/K374Kuv/aj+SpDE6ZjhU1XXzHFo/R9sCbpjnPHcAd8xRfwJYe6xxSJJOHn99hiSpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpM1I4JPm3SZ5O8lSS3UnOTHJRkseSzCS5O8kZre1b2/5MO75q6Dw3tfrXklwx2i1Jkka14HBIsgL418BkVa0FlgHXArcBt1fVu4BXgM2ty2bglVa/vbUjycWt37uBDcAnkyxb6LgkSaMbdVlpOXBWkuXAjwEvAu8H7m3HdwHXtO2NbZ92fH2StPpdVfW9qnoOmAEuHXFckqQRLDgcqmo/8J+AP2U2FF4DngRerapDrdk+YEXbXgG80Poeau3fOVyfo48kaQyWL7RjkvOY/Vf/RcCrwP9gdlloySTZAmwBmJiYYDAYLOXl9DfQ29Zs5ad3bV36C+06dpNRvG0NDAZnL+1F9Ka24HAA/jHwXFV9CyDJ54D3AecmWd5mByuB/a39fuBCYF9bhjoH+M5Q/YjhPj+iqnYAOwAmJydrampqhOHrzejPt97KN269ekmvMRgMWOrP5qqtDzC1aWmvoTe3UZ45/ClwWZIfa88O1gPPANPAB1ubTcB9bfv+tk87/vmqqla/tr3NdBGwGvjCCOOSJI1owTOHqnosyb3AF4FDwJeY/Vf9A8BdSW5ptZ2ty07gd5PMAAeYfUOJqno6yT3MBssh4IaqOrzQcUmSRjfKshJVdTNw81HlZ5njbaOq+kvg5+Y5z3Zg+yhjkSQtHr8hLUnqGA6SpI7hIEnqGA6SpM5ID6Sl09GqrQ8s/UUeWtprnHPWW5b0/JLhoDeVpf4CHMyGz8m4jrSUXFaSJHWcOUjHMPsLAE6wz20nfp3ZXxggnRqcOUjHUFUn9DM9PX3CfQwGnWoMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSZ6RwSHJuknuT/HGSvUl+Jsk7kuxJ8vX253mtbZL8RpKZJF9JcsnQeTa19l9PsmnUm5IkjWbUmcMngIeq6u8A7wH2AluBR6pqNfBI2we4EljdfrYAnwJI8g7gZuC9wKXAzUcCRZI0HgsOhyTnAP8I2AlQVX9VVa8CG4Fdrdku4Jq2vRG4s2Y9Cpyb5ALgCmBPVR2oqleAPcCGhY5LkjS6UWYOFwHfAn4nyZeSfDrJ2cBEVb3Y2rwETLTtFcALQ/33tdp8dUnSmCwfse8lwI1V9ViST/DDJSQAqqqSLNr/HDfJFmaXpJiYmGAwGCzWqaVFc/DgQT+bOu2NEg77gH1V9Vjbv5fZcPhmkguq6sW2bPRyO74fuHCo/8pW2w9MHVUfzHXBqtoB7ACYnJysqampuZpJYzUYDPCzqdPdgpeVquol4IUkP9VK64FngPuBI28cbQLua9v3Ax9uby1dBrzWlp8eBi5Pcl57EH15q0mSxmSUmQPAjcBnk5wBPAtcz2zg3JNkM/A88KHW9kHgKmAGeL21paoOJPkY8Hhr99GqOjDiuCRJIxgpHKrqy8DkHIfWz9G2gBvmOc8dwB2jjEWStHj8hrQkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6I4dDkmVJvpTkf7b9i5I8lmQmyd1Jzmj1t7b9mXZ81dA5bmr1ryW5YtQxSZJGsxgzh48Ae4f2bwNur6p3Aa8Am1t9M/BKq9/e2pHkYuBa4N3ABuCTSZYtwrgkSQs0UjgkWQlcDXy67Qd4P3Bva7ILuKZtb2z7tOPrW/uNwF1V9b2qeg6YAS4dZVySpNGMOnP4deCXgL9u++8EXq2qQ21/H7Ciba8AXgBox19r7X9Qn6OPJGkMli+0Y5IPAC9X1ZNJphZvSG94zS3AFoCJiQkGg8HJuKx0Qg4ePOhnU6e9BYcD8D7gZ5NcBZwJvB34BHBukuVtdrAS2N/a7wcuBPYlWQ6cA3xnqH7EcJ8fUVU7gB0Ak5OTNTU1NcLwpaUxGAzws6nT3YKXlarqpqpaWVWrmH2g/Pmq+mfANPDB1mwTcF/bvr/t045/vqqq1a9tbzNdBKwGvrDQcUmSRjfKzGE+vwzcleQW4EvAzlbfCfxukhngALOBQlU9neQe4BngEHBDVR1egnFJko7TooRDVQ2AQdt+ljneNqqqvwR+bp7+24HtizEWSdLo/Ia0JKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOgsOhyQXJplO8kySp5N8pNXfkWRPkq+3P89r9ST5jSQzSb6S5JKhc21q7b+eZNPotyVJGsUoM4dDwL+vqouBy4AbklwMbAUeqarVwCNtH+BKYHX72QJ8CmbDBLgZeC9wKXDzkUCRJI3HgsOhql6sqi+27T8H9gIrgI3ArtZsF3BN294I3FmzHgXOTXIBcAWwp6oOVNUrwB5gw0LHJUka3aI8c0iyCvh7wGPARFW92A69BEy07RXAC0Pd9rXafHVJ0pgsH/UESX4c+D3g31TVnyX5wbGqqiQ16jWGrrWF2SUpJiYmGAwGi3VqadEcPHjQz6ZOeyOFQ5K3MBsMn62qz7XyN5NcUFUvtmWjl1t9P3DhUPeVrbYfmDqqPpjrelW1A9gBMDk5WVNTU3M1k8ZqMBjgZ1Onu1HeVgqwE9hbVf956ND9wJE3jjYB9w3VP9zeWroMeK0tPz0MXJ7kvPYg+vJWkySNySgzh/cBPw98NcmXW+0/ALcC9yTZDDwPfKgdexC4CpgBXgeuB6iqA0k+Bjze2n20qg6MMC5J0ogWHA5V9b+BzHN4/RztC7hhnnPdAdyx0LFIkhaX35CWJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lS55QJhyQbknwtyUySreMejyS9mZ0S4ZBkGfBbwJXAxcB1SS4e76ikE7N7927Wrl3L+vXrWbt2Lbt37x73kKQFWz7uATSXAjNV9SxAkruAjcAzYx2VdJx2797Ntm3b2LlzJ4cPH2bZsmVs3rwZgOuuu27Mo5NO3CkxcwBWAC8M7e9rNem0sH37dnbu3Mm6detYvnw569atY+fOnWzfvn3cQ5MW5FSZORyXJFuALQATExMMBoPxDkhq9u7dy+HDhxkMBhw8eJDBYMDhw4fZu3evn1Odlk6VcNgPXDi0v7LVfkRV7QB2AExOTtbU1NRJGZx0LGvWrGHZsmVMTU0xGAyYmppienqaNWvW4OdUp6NTZVnpcWB1kouSnAFcC9w/5jFJx23btm1s3ryZ6elpDh06xPT0NJs3b2bbtm3jHpq0IKfEzKGqDiX5ReBhYBlwR1U9PeZhScftyEPnG2+8kb1797JmzRq2b9/uw2idtlJV4x7DgkxOTtYTTzwx7mFInSPLStKpKMmTVTV5rHanyrKSJOkUYjhIkjqGgySpYzhIkjqGgySpc9q+rZTkW8Dz4x6HNIfzgW+PexDSPP52Vf3EsRqdtuEgnaqSPHE8rwpKpzKXlSRJHcNBktQxHKTFt2PcA5BG5TMHSVLHmYMkqWM4SIsgyWeSfHDc45AWi+EgjUGSU+LX5UvzMRykeSQ5O8kDSf4oyVNJ/mmSX03yeNvfkSRz9JuzTZJBkl9P8gSwLclzSd7Sjr19eF8aN8NBmt8G4P9V1Xuqai3wEPCbVfUP2v5ZwAfm6PdGbc6oqsmq+jVgAFzd6tcCn6uq7y/VzUgnwnCQ5vdV4J8kuS3JP6yq14B1SR5L8lXg/cC75+j3Rm3uHtr+NHB9274e+J3FvwVpYVz3lOZRVX+S5BLgKuCWJI8ANwCTVfVCkv8InDncJ8mZwCffoM13h87/f5KsSjIFLKuqp5b0hqQT4MxBmkeSnwRer6r/BnwcuKQd+naSHwfmejvpzONoM+xO4L/jrEGnGGcO0vx+Gvh4kr8Gvg/8S+Aa4CngJeDxoztU1atJfvuN2hzls8AtwO5FHLc0Mr8hLY1R+27Exqr6+XGPRRrmzEEakyT/BbiS2Wca0inFmYMkqeMDaUlSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHX+Py52T43xU7imAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df[['salary']].boxplot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 重复值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### duplicated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3    False\n",
       "4    False\n",
       "5     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated()     # 默认只有一组中所有数据都与另一组相同，才会被认为是重复值    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jack</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7300</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lily</td>\n",
       "      <td>21.0</td>\n",
       "      <td>11000</td>\n",
       "      <td>python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tom</td>\n",
       "      <td>25.0</td>\n",
       "      <td>12000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Joe</td>\n",
       "      <td>81.0</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Jack</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   name   age  salary    lang\n",
       "0  Jack  18.0    8000  Python\n",
       "1  Mary   NaN    7300    Java\n",
       "2  Lily  21.0   11000  python\n",
       "3   Tom  25.0   12000     NaN\n",
       "4   Joe  81.0      -1        \n",
       "5  Jack  18.0    8000  Python"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jack</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7300</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lily</td>\n",
       "      <td>21.0</td>\n",
       "      <td>11000</td>\n",
       "      <td>python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tom</td>\n",
       "      <td>25.0</td>\n",
       "      <td>12000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Joe</td>\n",
       "      <td>81.0</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Jack</td>\n",
       "      <td>19.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   name   age  salary    lang\n",
       "0  Jack  18.0    8000  Python\n",
       "1  Mary   NaN    7300    Java\n",
       "2  Lily  21.0   11000  python\n",
       "3   Tom  25.0   12000     NaN\n",
       "4   Joe  81.0      -1        \n",
       "5  Jack  19.0    8000  Python"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[5, 'age'] = 19\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3    False\n",
       "4    False\n",
       "5    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3    False\n",
       "4    False\n",
       "5     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated(subset='lang')    # 只要lang这一列相同就会被认为是重复值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一致性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>age</th>\n",
       "      <th>salary</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jack</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7300</td>\n",
       "      <td>Java</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lily</td>\n",
       "      <td>21.0</td>\n",
       "      <td>11000</td>\n",
       "      <td>python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tom</td>\n",
       "      <td>25.0</td>\n",
       "      <td>12000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Joe</td>\n",
       "      <td>81.0</td>\n",
       "      <td>-1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Jack</td>\n",
       "      <td>19.0</td>\n",
       "      <td>8000</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   name   age  salary    lang\n",
       "0  Jack  18.0    8000  Python\n",
       "1  Mary   NaN    7300    Java\n",
       "2  Lily  21.0   11000  python\n",
       "3   Tom  25.0   12000     NaN\n",
       "4   Joe  81.0      -1        \n",
       "5  Jack  19.0    8000  Python"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Python', 'Java', 'python', nan, ''], dtype=object)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.lang.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上面的两个Python，一个是Python，一个是python，这就是数据不一致，导致筛选重复值时漏过"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
